package com.sinaapp.gavinzhang.GSpider.process;

import com.sinaapp.gavinzhang.GSpider.config.Constants;
import com.sinaapp.gavinzhang.GSpider.config.PageProcessorConfig;
import com.sinaapp.gavinzhang.GSpider.core.AbstractGWebUrlList;
import com.sinaapp.gavinzhang.GSpider.core.AbstractPageProcessor;
import com.sinaapp.gavinzhang.GSpider.core.AbstractProcess;
import com.sinaapp.gavinzhang.GSpider.core.PageProcessor;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * AbstractProcess类的实现
 * 流程：
 * websiteCheckAndDispose
 * 当 (网址为空 || 已经爬过 || http状态码有误)将网址置空返回。
 * 将网址放入已爬列表。
 * 得到网页，如果出错则将html和wenurl置空。
 *
 * 以下函数都将判断html是否为空
 *
 * preWebContent
 * 将得到的页面经由用户处理，返回处理国的Elements。
 *
 * disposeWebContent
 * 抓取符合规则的网址，放入临时网址列表
 *
 * afterWebContent
 * 将原始网页交由用户处理。
 * 将用额外加入的网址放入临时网址列表。
 * 将临时列表的网址置入待爬列表。
 * 
 * @author gavin
 * @version 1.0.0
 * @see com.sinaapp.gavinzhang.GSpider.core.AbstractProcess
 */
public class Process extends AbstractProcess {

	private static Logger log = Logger.getLogger(Process.class);

	private String webUrl;
	private Document html;
	private Elements elements;
	private boolean starting;

	private List<Pattern> patterns;
	private PageProcessor pp;
    private AbstractGWebUrlList pendingUrlList;
    private AbstractGWebUrlList visitedUrlList;
	private HttpProcessorInterceptor httpProcessorInterceptor;

	private PageProcessorConfig pageProcessorConfig;


	protected List<String> urllist = new ArrayList<>();

	public void run() {
		while (!Thread.currentThread().isInterrupted()) {
			this.webUrl = pendingUrlList.get();
			/** 判断网址是否为空，是否抓过，网站状态检查，下载*/
			websiteCheckAndDispose(webUrl);
			/** 预处理页面，从用户处得到处理后的elements，并且只从elements中爬取规则*/
			preWebContent(html);
			/** 从页面中得到可用的网址，存入list*/
			disposeWebContent(elements);
			/** 将原始网页交由用户处理。添加从用户得到网址*/
			afterWebContent(html);
		}
	}

	@Override
	public void websiteCheckAndDispose(String webUrl) {
		if(webUrl == null)return;
		/** 检查网址是否已经访问，已访问则直接返回*/
        if (visitedUrlList.exist(webUrl)) {
            this.webUrl = null;
            return;
        }
		/** 首先将这个网址加入已访问列表*/
		visitedUrlList.add(this.webUrl);

		HttpProcessor httpProcessor = new HttpProcessor(webUrl);
		int status = httpProcessor.getStatus();
		/**判断http返回状态码，大于400 直接返回 bug：429为请求太多 503服务器主动拒绝*/
		switch (status) {
			case 200:break;
			case 403:
			case 404:
			case 500:
				log.error(Constants.SERVICE_ERROR + this.webUrl);
				this.webUrl = null;
				html = null;
				return;
			default:break;
		}

		/** 最多timeout次判断 当status为429或200时
		 *  去掉(status==429 || status==503 || status==200) &&
		 * */
		int timeout = pageProcessorConfig.getTimeout();
		int sleepTime = pageProcessorConfig.getSleepTime();

		while (--timeout>0) {
			try {
				//TODO use list in future
				httpProcessorInterceptor.preProcess(httpProcessor);
				html = Jsoup.parse(httpProcessor.getContent(),webUrl);
				httpProcessorInterceptor.afterProcess(httpProcessor);

				timeout = 0;
			} catch (IOException e) {
				//协议 网址 连接出错
				//e.printStackTrace();
			}
			/** 如果不等于0 则说明没有拿到网页数据*/
			if (timeout != 0) {
				log.warn("超时重试次数：" + (pageProcessorConfig.getTimeout() - timeout) + " " + this.webUrl);
				status = httpProcessor.getStatus();
				if (status==429 || status==503) {
					try {
						Thread.sleep(sleepTime);
						sleepTime = sleepTime+pageProcessorConfig.getWaitTime();
					} catch (InterruptedException e) {
						//中断结束
						break;
					}
				}
			}
		}
		if (html == null) {
			log.error(Constants.CATCH_URL_ERROR + this.webUrl);
		}
	}


	@Override
	public void preWebContent(Document html) {
		if(html == null)return;
		/** 从用户处得到要处理的页面*/
		Elements tmp = pp.preDispose(html);
		if (tmp != null){
			elements = tmp;
		}else {
			elements = html.select("body");
		}
	}

	@Override
	public void disposeWebContent(Elements html) {
		/** 如果html为空则直接返回*/
		if(html == null)return;
		Elements elements =  this.elements.select("a");
		/** 如果正则表达式列表没有设置则有问题.由GSpider.check方法检查*/
        //if(listP==null || listP.size()==0)return;
		/** 的到符合规则的网址，存入临时列表*/
        for (Pattern pa : patterns) {
            for (Element e : elements) {
                if (e.absUrl("href").matches(pa.toString())){
					urllist.add(e.absUrl("href"));
                }
            }
         }
	}

	@Override
	public void afterWebContent(Document html) {
		/** 在chenkanddown 后都用这个来判断是否有问题*/
		if(html == null)return;
		/** 先添加找到的网址，后再加用户找到的*/
		if(pendingUrlList.size() > Constants.CHECK_VISITEDLIST_SIZE) {
			//同步处理
			synchronized(visitedUrlList) {
				for (String url : urllist) {
					if(!visitedUrlList.exist(url)) {
						pendingUrlList.add(url);
					}
				}
			}
		}else {
			for (String url : urllist) {
				pendingUrlList.add(url);
			}
		}
		if (urllist.size() != 0) {
			urllist = new ArrayList<>();
		}

		//urllist.clear();
		/** 用户处理页面*/
		pp.dispose(html, webUrl);
		/** 得到额外的网址列表,暂时不能使用*/
		List<String> tmp = pp.getUrlList();
		if (tmp!=null && tmp.size()!=0){
			if(visitedUrlList.size() > Constants.CHECK_VISITEDLIST_SIZE) {
				for (String url : tmp) {
					if(!visitedUrlList.exist(url))
					{
						pendingUrlList.add(url);
					}
				}
			}else {
				for (String url : tmp) {
					pendingUrlList.add(url);
				}
			}
		}
		webUrl = null;
		this.html = null;
	}

	public void setPageProcessorConfig(PageProcessorConfig pageProcessorConfig)
	{
		this.pageProcessorConfig = pageProcessorConfig;
		this.pendingUrlList = pageProcessorConfig.getCurrentUrlList();
		this.pp = pageProcessorConfig.getPageProcessor();
		this.visitedUrlList = pageProcessorConfig.getFoundUrlList();
		this.patterns = pageProcessorConfig.getPatternUrls();
		this.httpProcessorInterceptor = pageProcessorConfig.getHttpProcessorInterceptor();
	}

	@Override
	public void setPendingUrlList(AbstractGWebUrlList list) {
		this.pendingUrlList = list;
	}

	@Override
	public void setVisitedUrlList(AbstractGWebUrlList list) {
		this.visitedUrlList = list;
	}

	public Elements getElements() {
		return elements;
	}

	public void setElements(Elements elements) {
		this.elements = elements;
	}

	public AbstractGWebUrlList getCurrentUrlList() {
		return pendingUrlList;
	}

	public void setCurrentUrlList(AbstractGWebUrlList currentUrlList) {
		this.pendingUrlList = currentUrlList;
	}

	@Override
	public void setWebUrl(String webUrl) {
		this.webUrl = webUrl;
	}

	@Override
	public void setPatterns(List<Pattern> patterns) {
		this.patterns = patterns;
	}

	@Override
	public void setPageProcessor(AbstractPageProcessor pp) {
		this.pp = pp;
	}

	public boolean isStarting() {
		return starting;
	}

	public void setStarting(boolean starting) {
		this.starting = starting;
	}

	public void setHttpProcessorInterceptor(HttpProcessorInterceptor httpProcessorInterceptor) {
		this.httpProcessorInterceptor = httpProcessorInterceptor;
	}
}
